{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Semi-supervised Self-training Classification with mnist Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.1\n"
     ]
    }
   ],
   "source": [
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = tf.keras.datasets.mnist\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "x_train, x_test = x_train / 255.0, x_test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train.shape = (60000, 28, 28)\n",
      "y_train.shape = (60000,)\n",
      "x_test.shape = (10000, 28, 28)\n",
      "y_test.shape = (10000,)\n"
     ]
    }
   ],
   "source": [
    "print(\"x_train.shape = {}\".format(x_train.shape))\n",
    "print(\"y_train.shape = {}\".format(y_train.shape))\n",
    "print(\"x_test.shape = {}\".format(x_test.shape))\n",
    "print(\"y_test.shape = {}\".format(y_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "HEIGHT = 28\n",
    "WIDTH = 28\n",
    "NCLASSES = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.eye(N = NCLASSES)[y_train]\n",
    "y_test = np.eye(N = NCLASSES)[y_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train.shape = (60000, 28, 28)\n",
      "y_train.shape = (60000, 10)\n",
      "x_test.shape = (10000, 28, 28)\n",
      "y_test.shape = (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "print(\"x_train.shape = {}\".format(x_train.shape))\n",
    "print(\"y_train.shape = {}\".format(y_train.shape))\n",
    "print(\"x_test.shape = {}\".format(x_test.shape))\n",
    "print(\"y_test.shape = {}\".format(y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create fully supervised model for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "  x = {\"image\": x_train},\n",
    "  y = y_train,\n",
    "  batch_size = 100,\n",
    "  num_epochs = None,\n",
    "  shuffle = True,\n",
    "  queue_capacity = 5000)\n",
    "\n",
    "eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "  x = {\"image\": x_test},\n",
    "  y = y_test,\n",
    "  batch_size = 100,\n",
    "  num_epochs = 1,\n",
    "  shuffle = False,\n",
    "  queue_capacity = 5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_model(img, mode, hparams):\n",
    "  X = tf.reshape(tensor = img, shape = [-1,HEIGHT * WIDTH])  # flatten\n",
    "  ylogits = tf.layers.dense(inputs = X, units = NCLASSES, activation = None)\n",
    "  return ylogits, NCLASSES\n",
    "\n",
    "def dnn_model(img, mode, hparams):\n",
    "  X = tf.reshape(tensor = img, shape = [-1, HEIGHT * WIDTH])  # flatten\n",
    "  h1 = tf.layers.dense(inputs = X, units = 300, activation = tf.nn.relu)\n",
    "  h2 = tf.layers.dense(inputs = h1, units = 100, activation = tf.nn.relu)\n",
    "  h3 = tf.layers.dense(inputs = h2, units = 30, activation = tf.nn.relu)\n",
    "  ylogits = tf.layers.dense(inputs = h3, units = NCLASSES, activation = None)\n",
    "  return ylogits, NCLASSES\n",
    "\n",
    "def dnn_dropout_model(img, mode, hparams):\n",
    "  dprob = hparams.get(\"dprob\", 0.1)\n",
    "\n",
    "  X = tf.reshape(tensor = img, shape = [-1, HEIGHT * WIDTH])  #flatten\n",
    "  h1 = tf.layers.dense(inputs = X, units = 300, activation = tf.nn.relu)\n",
    "  h2 = tf.layers.dense(inputs = h1, units = 100, activation = tf.nn.relu)\n",
    "  h3 = tf.layers.dense(inputs = h2, units = 30, activation = tf.nn.relu)\n",
    "  h3d = tf.layers.dropout(\n",
    "    inputs = h3, \n",
    "    rate = dprob, \n",
    "    training = (mode == tf.estimator.ModeKeys.TRAIN))  # only dropout when training\n",
    "  ylogits = tf.layers.dense(inputs = h3d, units = NCLASSES, activation = None)\n",
    "  return ylogits, NCLASSES\n",
    "\n",
    "def cnn_model(img, mode, hparams):\n",
    "  ksize1 = hparams.get(\"ksize1\", 5)\n",
    "  ksize2 = hparams.get(\"ksize2\", 5)\n",
    "  nfil1 = hparams.get(\"nfil1\", 10)\n",
    "  nfil2 = hparams.get(\"nfil2\", 20)\n",
    "  dprob = hparams.get(\"dprob\", 0.25)\n",
    "\n",
    "  c1 = tf.layers.conv2d(inputs = img, filters = nfil1,\n",
    "              kernel_size = ksize1, strides = 1, # ?x28x28x10\n",
    "              padding = \"same\", activation = tf.nn.relu)\n",
    "  p1 = tf.layers.max_pooling2d(inputs = c1, pool_size = 2, strides = 2)  # ?x14x14x10\n",
    "  c2 = tf.layers.conv2d(inputs = p1, filters = nfil2,\n",
    "              kernel_size = ksize2, strides = 1, \n",
    "              padding = \"same\", activation = tf.nn.relu)\n",
    "  p2 = tf.layers.max_pooling2d(inputs = c2, pool_size = 2, strides = 2)  # ?x7x7x20\n",
    "  \n",
    "  outlen = p2.shape[1] * p2.shape[2] * p2.shape[3] #980\n",
    "  p2flat = tf.reshape(tensor = p2, shape = [-1, outlen]) # flattened\n",
    "\n",
    "  # Apply batch normalization\n",
    "  if hparams[\"batch_norm\"]:\n",
    "    h3 = tf.layers.dense(inputs = p2flat, units = 300, activation = None)\n",
    "    h3 = tf.layers.batch_normalization(\n",
    "      x = h3, \n",
    "      training = (mode == tf.estimator.ModeKeys.TRAIN))  # only batchnorm when training\n",
    "    h3 = tf.nn.relu(x = h3)\n",
    "  else:  \n",
    "    h3 = tf.layers.dense(inputs = p2flat, units = 300, activation = tf.nn.relu)\n",
    "  \n",
    "  # Apply dropout\n",
    "  h3d = tf.layers.dropout(\n",
    "    inputs = h3, rate = dprob, training = (mode == tf.estimator.ModeKeys.TRAIN))\n",
    "\n",
    "  ylogits = tf.layers.dense(inputs = h3d, units = NCLASSES, activation = None)\n",
    "    \n",
    "  # Apply batch normalization once more\n",
    "  if hparams[\"batch_norm\"]:\n",
    "     ylogits = tf.layers.batch_normalization(\n",
    "       x = ylogits, \n",
    "       training = (mode == tf.estimator.ModeKeys.TRAIN))\n",
    "\n",
    "  return ylogits, NCLASSES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def image_classifier(features, labels, mode, params):\n",
    "  print(\"\\nfeatures = \\n{}\".format(features))\n",
    "  print(\"labels = \\n{}\".format(labels))\n",
    "  print(\"mode = \\n{}\".format(mode))\n",
    "  print(\"params = \\n{}\".format(params))\n",
    "  \n",
    "  model_functions = {\n",
    "    \"linear\":linear_model,\n",
    "    \"dnn\":dnn_model,\n",
    "    \"dnn_dropout\":dnn_dropout_model,\n",
    "    \"cnn\":cnn_model}\n",
    "  \n",
    "  model_function = model_functions[params[\"model\"]]  \n",
    "  \n",
    "  ylogits, nclasses = model_function(features[\"image\"], mode, params)\n",
    "  print(\"ylogits = \\n{}\".format(ylogits))\n",
    "  probabilities = tf.nn.softmax(logits = ylogits)  # shape = (current_batch_size, NCLASSES)\n",
    "  print(\"probabilities = \\n{}\".format(probabilities))\n",
    "  class_ids = tf.cast(\n",
    "    x = tf.argmax(\n",
    "      input = probabilities, axis = 1), dtype = tf.uint8)  # shape = (current_batch_size,)\n",
    "  print(\"class_ids = \\n{}\".format(class_ids))\n",
    "  \n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    loss = tf.reduce_mean(\n",
    "      input_tensor = tf.nn.softmax_cross_entropy_with_logits_v2(\n",
    "        logits = ylogits, labels = labels))\n",
    "    eval_metric_ops = {\n",
    "      \"accuracy\": tf.metrics.accuracy(\n",
    "        labels = tf.argmax(input = labels, axis = 1), predictions = class_ids)}\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      # This is needed for batch normalization, but has no effect otherwise\n",
    "      update_ops = tf.get_collection(key = tf.GraphKeys.UPDATE_OPS)\n",
    "      with tf.control_dependencies(update_ops):\n",
    "        train_op = tf.contrib.layers.optimize_loss(\n",
    "          loss = loss, \n",
    "          global_step = tf.train.get_global_step(),\n",
    "          learning_rate = params[\"learning_rate\"], \n",
    "          optimizer = \"Adam\")\n",
    "    else:\n",
    "      train_op = None\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    " \n",
    "  return tf.estimator.EstimatorSpec(\n",
    "    mode = mode,\n",
    "    predictions = {\"probabilities\": probabilities, \"class_ids\": class_ids},\n",
    "    loss = loss,\n",
    "    train_op = train_op,\n",
    "    eval_metric_ops = eval_metric_ops,\n",
    "    export_outputs = {\n",
    "      \"classes\": tf.estimator.export.PredictOutput(\n",
    "        {\"probabilities\": probabilities, \n",
    "         \"class_ids\": class_ids})})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def serving_input_fn():\n",
    "  # Input will be rank 3\n",
    "  feature_placeholders = {\n",
    "    \"image\": tf.placeholder(dtype = tf.float64, shape = [None, HEIGHT, WIDTH])}\n",
    "  # But model function requires rank 4\n",
    "  features = {\n",
    "    \"image\": tf.expand_dims(input = feature_placeholders[\"image\"], axis = -1)} \n",
    "  return tf.estimator.export.ServingInputReceiver(\n",
    "    features = features, \n",
    "    receiver_tensors = feature_placeholders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_evaluate(output_dir, hparams):\n",
    "  # Ensure filewriter cache is clear for TensorBoard events file\n",
    "  tf.summary.FileWriterCache.clear()\n",
    "  EVAL_INTERVAL = 60\n",
    "\n",
    "  supervised_estimator = tf.estimator.Estimator(\n",
    "    model_fn = image_classifier,\n",
    "    params = hparams,\n",
    "    config = tf.estimator.RunConfig(\n",
    "      save_checkpoints_secs = EVAL_INTERVAL),\n",
    "    model_dir = output_dir)\n",
    "  \n",
    "  train_spec = tf.estimator.TrainSpec(\n",
    "    input_fn = train_input_fn,\n",
    "    max_steps = hparams[\"train_steps\"])\n",
    "  \n",
    "  exporter = tf.estimator.LatestExporter(\n",
    "    name = \"exporter\", \n",
    "    serving_input_receiver_fn = serving_input_fn)\n",
    "  \n",
    "  eval_spec = tf.estimator.EvalSpec(\n",
    "    input_fn = eval_input_fn,\n",
    "    steps = None,\n",
    "    exporters = exporter,\n",
    "    throttle_secs = EVAL_INTERVAL)\n",
    "  \n",
    "  tf.estimator.train_and_evaluate(\n",
    "    estimator = supervised_estimator, \n",
    "    train_spec = train_spec, \n",
    "    eval_spec = eval_spec)\n",
    "  \n",
    "  return supervised_estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams = {}\n",
    "hparams[\"train_batch_size\"] = 100\n",
    "hparams[\"learning_rate\"] = 0.01\n",
    "hparams[\"train_steps\"] = 1000\n",
    "hparams[\"ksize1\"] = 5\n",
    "hparams[\"ksize2\"] = 5\n",
    "hparams[\"nfil1\"] = 10\n",
    "hparams[\"nfil2\"] = 20\n",
    "hparams[\"dprob\"] = 0.1\n",
    "hparams[\"batch_norm\"] = False\n",
    "hparams[\"model\"] = \"linear\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_service': None, '_device_fn': None, '_task_type': 'worker', '_evaluation_master': '', '_eval_distribute': None, '_is_chief': True, '_protocol': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_save_summary_steps': 100, '_keep_checkpoint_every_n_hours': 10000, '_save_checkpoints_secs': 60, '_tf_random_seed': None, '_master': '', '_log_step_count_steps': 100, '_save_checkpoints_steps': None, '_global_id_in_cluster': 0, '_model_dir': 'supervised_trained', '_keep_checkpoint_max': 5, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f5dd805bcf8>, '_task_id': 0, '_train_distribute': None, '_experimental_distribute': None}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 60.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'random_shuffle_queue_DequeueMany:1' shape=(100, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"random_shuffle_queue_DequeueMany:2\", shape=(100, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "train\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "WARNING:tensorflow:From <ipython-input-8-902f303df411>:3: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(100, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(100, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(100,), dtype=uint8)\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py:809: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.441912885957754, step = 0\n",
      "INFO:tensorflow:global_step/sec: 321.814\n",
      "INFO:tensorflow:loss = 0.2767097747695082, step = 100 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 348.453\n",
      "INFO:tensorflow:loss = 0.3221954010373979, step = 200 (0.285 sec)\n",
      "INFO:tensorflow:global_step/sec: 353.258\n",
      "INFO:tensorflow:loss = 0.3627534561186343, step = 300 (0.284 sec)\n",
      "INFO:tensorflow:global_step/sec: 345.939\n",
      "INFO:tensorflow:loss = 0.2781262124740381, step = 400 (0.288 sec)\n",
      "INFO:tensorflow:global_step/sec: 358.863\n",
      "INFO:tensorflow:loss = 0.2557627886139751, step = 500 (0.278 sec)\n",
      "INFO:tensorflow:global_step/sec: 322.861\n",
      "INFO:tensorflow:loss = 0.5794160362105137, step = 600 (0.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 343.111\n",
      "INFO:tensorflow:loss = 0.28384661502428254, step = 700 (0.291 sec)\n",
      "INFO:tensorflow:global_step/sec: 341.737\n",
      "INFO:tensorflow:loss = 0.3293382458190204, step = 800 (0.295 sec)\n",
      "INFO:tensorflow:global_step/sec: 335.711\n",
      "INFO:tensorflow:loss = 0.2702323930142859, step = 900 (0.297 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"fifo_queue_DequeueUpTo:2\", shape=(?, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "eval\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-15T23:55:42Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from supervised_trained/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-15-23:55:42\n",
      "INFO:tensorflow:Saving dict for global step 1000: accuracy = 0.9118, global_step = 1000, loss = 0.31161386\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: supervised_trained/model.ckpt-1000\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'ExpandDims:0' shape=(?, 28, 28, 1) dtype=float64>}\n",
      "labels = \n",
      "None\n",
      "mode = \n",
      "infer\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default', 'classes']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from supervised_trained/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: supervised_trained/export/exporter/temp-b'1552694142'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.5330232727938002.\n"
     ]
    }
   ],
   "source": [
    "SUPERVISED_MODEL_DIR = \"supervised_trained\"\n",
    "shutil.rmtree(path = SUPERVISED_MODEL_DIR, ignore_errors = True)  # start fresh each time\n",
    "supervised_estimator = train_and_evaluate(SUPERVISED_MODEL_DIR, hparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"fifo_queue_DequeueUpTo:2\", shape=(?, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "eval\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-15T23:55:42Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from supervised_trained/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-15-23:55:43\n",
      "INFO:tensorflow:Saving dict for global step 1000: accuracy = 0.9118, global_step = 1000, loss = 0.31161386\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: supervised_trained/model.ckpt-1000\n"
     ]
    }
   ],
   "source": [
    "eval_metrics = supervised_estimator.evaluate(input_fn = eval_input_fn, steps = None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now create semi-supervised model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "percent_labeled = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number_of_train_examples = 60000\n"
     ]
    }
   ],
   "source": [
    "number_of_train_examples = x_train.shape[0]\n",
    "print(\"number_of_train_examples = {}\".format(number_of_train_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number_of_labeled_train_examples = 600 & number_of_unlabeled_train_examples = 59400\n"
     ]
    }
   ],
   "source": [
    "number_of_labeled_train_examples = int(number_of_train_examples * percent_labeled)\n",
    "number_of_unlabeled_train_examples = number_of_train_examples - number_of_labeled_train_examples\n",
    "print(\"number_of_labeled_train_examples = {} & number_of_unlabeled_train_examples = {}\".format(number_of_labeled_train_examples, number_of_unlabeled_train_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "semi_supervised_labeled_x_train_original_arr = x_train[0:number_of_labeled_train_examples]\n",
    "semi_supervised_labeled_y_train_original_arr = y_train[0:number_of_labeled_train_examples]\n",
    "semi_supervised_unlabeled_x_train_original_arr = x_train[number_of_labeled_train_examples:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "semi_supervised_labeled_x_train_original_arr.shape = (600, 28, 28)\n",
      "semi_supervised_labeled_y_train_original_arr.shape = (600, 10)\n",
      "semi_supervised_unlabeled_x_train_original_arr.shape = (59400, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "print(\"semi_supervised_labeled_x_train_original_arr.shape = {}\".format(semi_supervised_labeled_x_train_original_arr.shape))\n",
    "print(\"semi_supervised_labeled_y_train_original_arr.shape = {}\".format(semi_supervised_labeled_y_train_original_arr.shape))\n",
    "print(\"semi_supervised_unlabeled_x_train_original_arr.shape = {}\".format(semi_supervised_unlabeled_x_train_original_arr.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create semi-supervised model using sparse labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEMI_SUPERVISED_MODEL_DIR = \"semi_supervised_trained\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_service': None, '_device_fn': None, '_task_type': 'worker', '_evaluation_master': '', '_eval_distribute': None, '_is_chief': True, '_protocol': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_save_summary_steps': 100, '_keep_checkpoint_every_n_hours': 10000, '_save_checkpoints_secs': 30, '_tf_random_seed': None, '_master': '', '_log_step_count_steps': 100, '_save_checkpoints_steps': None, '_global_id_in_cluster': 0, '_model_dir': 'semi_supervised_trained', '_keep_checkpoint_max': 5, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f5da6e157f0>, '_task_id': 0, '_train_distribute': None, '_experimental_distribute': None}\n"
     ]
    }
   ],
   "source": [
    "EVAL_INTERVAL = 30\n",
    "semi_supervised_estimator = tf.estimator.Estimator(\n",
    "  model_fn = image_classifier,\n",
    "  params = hparams,\n",
    "  config = tf.estimator.RunConfig(\n",
    "    save_checkpoints_secs = EVAL_INTERVAL),\n",
    "  model_dir = SEMI_SUPERVISED_MODEL_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "confidence_threshold = 0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "loop_counter = 0, number_of_labeled_examples = 600, number_of_unlabeled_examples = 59400\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'random_shuffle_queue_DequeueMany:1' shape=(32, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"random_shuffle_queue_DequeueMany:2\", shape=(32, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "train\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(32, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(32, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(32,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.365938087836863, step = 0\n",
      "INFO:tensorflow:global_step/sec: 399.401\n",
      "INFO:tensorflow:loss = 0.13523153655250514, step = 100 (0.252 sec)\n",
      "INFO:tensorflow:global_step/sec: 456.058\n",
      "INFO:tensorflow:loss = 0.02680597472577697, step = 200 (0.222 sec)\n",
      "INFO:tensorflow:global_step/sec: 445.282\n",
      "INFO:tensorflow:loss = 0.025236673435424586, step = 300 (0.224 sec)\n",
      "INFO:tensorflow:global_step/sec: 455.727\n",
      "INFO:tensorflow:loss = 0.016117166623450287, step = 400 (0.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 458.466\n",
      "INFO:tensorflow:loss = 0.012395828387475948, step = 500 (0.218 sec)\n",
      "INFO:tensorflow:global_step/sec: 463.328\n",
      "INFO:tensorflow:loss = 0.012971538780879213, step = 600 (0.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 463.616\n",
      "INFO:tensorflow:loss = 0.006308051908256436, step = 700 (0.215 sec)\n",
      "INFO:tensorflow:global_step/sec: 462.46\n",
      "INFO:tensorflow:loss = 0.007308451547341978, step = 800 (0.219 sec)\n",
      "INFO:tensorflow:global_step/sec: 453.807\n",
      "INFO:tensorflow:loss = 0.0059257714200930105, step = 900 (0.221 sec)\n",
      "INFO:tensorflow:global_step/sec: 445.589\n",
      "INFO:tensorflow:loss = 0.0035080901940831165, step = 1000 (0.221 sec)\n",
      "INFO:tensorflow:global_step/sec: 454.284\n",
      "INFO:tensorflow:loss = 0.0028246791837094843, step = 1100 (0.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 457.513\n",
      "INFO:tensorflow:loss = 0.0024214909460709543, step = 1200 (0.219 sec)\n",
      "INFO:tensorflow:global_step/sec: 441.347\n",
      "INFO:tensorflow:loss = 0.0029520669214374686, step = 1300 (0.229 sec)\n",
      "INFO:tensorflow:global_step/sec: 447.536\n",
      "INFO:tensorflow:loss = 0.003128717154198998, step = 1400 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 460.051\n",
      "INFO:tensorflow:loss = 0.0017081914939195758, step = 1500 (0.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 456.885\n",
      "INFO:tensorflow:loss = 0.0020213854644541445, step = 1600 (0.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 447.012\n",
      "INFO:tensorflow:loss = 0.0021481597733560556, step = 1700 (0.227 sec)\n",
      "INFO:tensorflow:global_step/sec: 448.231\n",
      "INFO:tensorflow:loss = 0.0019354064116743769, step = 1800 (0.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 456.062\n",
      "INFO:tensorflow:loss = 0.001704114354353919, step = 1900 (0.219 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0011641851937206165.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"fifo_queue_DequeueUpTo:2\", shape=(?, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "eval\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-15T23:55:48Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-15-23:55:49\n",
      "INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.8502, global_step = 2000, loss = 0.7520157\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "None\n",
      "mode = \n",
      "infer\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "loop_counter = 1, number_of_labeled_examples = 38231, number_of_unlabeled_examples = 21769\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'random_shuffle_queue_DequeueMany:1' shape=(32, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"random_shuffle_queue_DequeueMany:2\", shape=(32, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "train\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(32, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(32, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(32,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-2000\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.01141419890458436, step = 2000\n",
      "INFO:tensorflow:global_step/sec: 411.224\n",
      "INFO:tensorflow:loss = 0.006622091477450838, step = 2100 (0.247 sec)\n",
      "INFO:tensorflow:global_step/sec: 448.973\n",
      "INFO:tensorflow:loss = 0.014475202875871479, step = 2200 (0.221 sec)\n",
      "INFO:tensorflow:global_step/sec: 452.369\n",
      "INFO:tensorflow:loss = 0.009824431950559624, step = 2300 (0.224 sec)\n",
      "INFO:tensorflow:global_step/sec: 454.137\n",
      "INFO:tensorflow:loss = 0.007456795809185349, step = 2400 (0.219 sec)\n",
      "INFO:tensorflow:global_step/sec: 452.366\n",
      "INFO:tensorflow:loss = 0.0031087645686271263, step = 2500 (0.219 sec)\n",
      "INFO:tensorflow:global_step/sec: 452.753\n",
      "INFO:tensorflow:loss = 0.0077568737218731395, step = 2600 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 429.369\n",
      "INFO:tensorflow:loss = 0.009810967225731923, step = 2700 (0.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 446.098\n",
      "INFO:tensorflow:loss = 0.011808617194771009, step = 2800 (0.226 sec)\n",
      "INFO:tensorflow:global_step/sec: 442.863\n",
      "INFO:tensorflow:loss = 0.006893941773865615, step = 2900 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 457.263\n",
      "INFO:tensorflow:loss = 0.008738697800364858, step = 3000 (0.219 sec)\n",
      "INFO:tensorflow:global_step/sec: 444.881\n",
      "INFO:tensorflow:loss = 0.006993524168726313, step = 3100 (0.227 sec)\n",
      "INFO:tensorflow:global_step/sec: 450.784\n",
      "INFO:tensorflow:loss = 0.006073000478958595, step = 3200 (0.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 436.294\n",
      "INFO:tensorflow:loss = 0.011409685457427536, step = 3300 (0.232 sec)\n",
      "INFO:tensorflow:global_step/sec: 443.752\n",
      "INFO:tensorflow:loss = 0.015676828116508505, step = 3400 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 439.319\n",
      "INFO:tensorflow:loss = 0.011115260386718898, step = 3500 (0.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 446.081\n",
      "INFO:tensorflow:loss = 0.012624634648589466, step = 3600 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 444.859\n",
      "INFO:tensorflow:loss = 0.008001528046230923, step = 3700 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 447.951\n",
      "INFO:tensorflow:loss = 0.004924472112049076, step = 3800 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 448.839\n",
      "INFO:tensorflow:loss = 0.007214280338301751, step = 3900 (0.223 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4000 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.007598015524044973.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"fifo_queue_DequeueUpTo:2\", shape=(?, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "eval\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-15T23:55:55Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-4000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-15-23:55:56\n",
      "INFO:tensorflow:Saving dict for global step 4000: accuracy = 0.8492, global_step = 4000, loss = 0.7550451\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 4000: semi_supervised_trained/model.ckpt-4000\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "None\n",
      "mode = \n",
      "infer\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-4000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "shutil.rmtree(path = SEMI_SUPERVISED_MODEL_DIR, ignore_errors = True) # start fresh each time\n",
    "\n",
    "semi_supervised_labeled_x_train_arr = semi_supervised_labeled_x_train_original_arr\n",
    "semi_supervised_labeled_y_train_arr = semi_supervised_labeled_y_train_original_arr\n",
    "semi_supervised_unlabeled_x_train_arr = semi_supervised_unlabeled_x_train_original_arr\n",
    "\n",
    "new_labeled_x_train_arr = np.zeros([1])\n",
    "\n",
    "accuracy = 0.000001\n",
    "old_accuracy = 0.0\n",
    "\n",
    "loop_counter = 0\n",
    "while semi_supervised_unlabeled_x_train_arr.shape[0] > 0 and new_labeled_x_train_arr.shape[0] > 0 and accuracy > old_accuracy:\n",
    "  print(\"\\nloop_counter = {}, number_of_labeled_examples = {}, number_of_unlabeled_examples = {}\\n\".format(loop_counter, semi_supervised_labeled_x_train_arr.shape[0], semi_supervised_unlabeled_x_train_arr.shape[0]))\n",
    "  # Train on currently labeled data\n",
    "  train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x = {\"image\": semi_supervised_labeled_x_train_arr}, \n",
    "    y = semi_supervised_labeled_y_train_arr, \n",
    "    batch_size = 32, \n",
    "    num_epochs = None, \n",
    "    shuffle = True)\n",
    "\n",
    "  semi_supervised_estimator.train(\n",
    "    input_fn = train_input_fn, \n",
    "    steps = 2000)\n",
    "\n",
    "\n",
    "  # Check evaluation metrics on held out evaluation set now that training is over\n",
    "  eval_metrics = semi_supervised_estimator.evaluate(\n",
    "    input_fn = eval_input_fn, \n",
    "    steps = None)\n",
    "  \n",
    "  old_accuracy = accuracy\n",
    "  accuracy = eval_metrics[\"accuracy\"]\n",
    "\n",
    "  # Now predict from the unlabeled set\n",
    "  predict_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x = {\"image\": semi_supervised_unlabeled_x_train_arr}, \n",
    "    y = None, \n",
    "    batch_size = 512, \n",
    "    num_epochs = 1, \n",
    "    shuffle = False)\n",
    "\n",
    "  predictions = [prediction \n",
    "                 for prediction in semi_supervised_estimator.predict(\n",
    "                   input_fn = predict_input_fn)]\n",
    "\n",
    "  # Get the probabilities from the prediction list generated from the estimator\n",
    "  probabilities = np.array(object = [prediction[\"probabilities\"] for prediction in predictions])\n",
    "\n",
    "  # Check if our predictions are above the confidence threshold\n",
    "  confidence_condition = np.amax(a = probabilities, axis = 1) > confidence_threshold\n",
    "\n",
    "  # Create array of the confidently prediction examples combining their features with the predicted probabilities\n",
    "  new_labeled_x_train_arr = semi_supervised_unlabeled_x_train_arr[confidence_condition]\n",
    "  new_labeled_y_train_arr = probabilities[confidence_condition]\n",
    "\n",
    "  semi_supervised_labeled_x_train_arr = np.concatenate(\n",
    "    seq = [semi_supervised_labeled_x_train_arr, new_labeled_x_train_arr], axis = 0)\n",
    "  semi_supervised_labeled_y_train_arr = np.concatenate(\n",
    "    seq = [semi_supervised_labeled_y_train_arr, new_labeled_y_train_arr], axis = 0)\n",
    "\n",
    "  # Remove the confident predictions leaving only the unconfident predictions to go another round through the loop\n",
    "  semi_supervised_unlabeled_x_train_arr = semi_supervised_unlabeled_x_train_arr[~confidence_condition]\n",
    "  \n",
    "  loop_counter += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use kmeans to improve results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First use PCA to reduce the dimensionality going into kmeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s = \n",
      "Tensor(\"Svd:0\", shape=(784,), dtype=float32)\n",
      "u = \n",
      "Tensor(\"Svd:1\", shape=(60000, 784), dtype=float32)\n",
      "v = \n",
      "Tensor(\"Svd:2\", shape=(784, 784), dtype=float32)\n",
      "sigma = \n",
      "Tensor(\"Diag:0\", shape=(784, 784), dtype=float32)\n",
      "x_train_pca = \n",
      "Tensor(\"MatMul:0\", shape=(60000, 10), dtype=float32)\n",
      "x_train_pca_arr.shape = \n",
      "(60000, 10)\n"
     ]
    }
   ],
   "source": [
    "number_of_dimensions = 10\n",
    "\n",
    "s, u, v = tf.svd(\n",
    "  tensor = tf.convert_to_tensor(\n",
    "    value = x_train.reshape([-1, HEIGHT * WIDTH]), \n",
    "    dtype = tf.float32), \n",
    "  full_matrices = False, \n",
    "  compute_uv = True)\n",
    "print(\"s = \\n{}\".format(s))\n",
    "print(\"u = \\n{}\".format(u))\n",
    "print(\"v = \\n{}\".format(v))\n",
    "\n",
    "sigma = tf.diag(diagonal = s)\n",
    "print(\"sigma = \\n{}\".format(sigma))\n",
    "\n",
    "x_train_pca = tf.matmul(a = u, b = sigma[:, 0:number_of_dimensions])\n",
    "print(\"x_train_pca = \\n{}\".format(x_train_pca))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "  x_train_pca_arr = sess.run(fetches = x_train_pca)\n",
    "print(\"x_train_pca_arr.shape = \\n{}\".format(x_train_pca_arr.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "KMEANS_MODEL_DIR = \"kmeans_estimator\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_service': None, '_device_fn': None, '_task_type': 'worker', '_evaluation_master': '', '_eval_distribute': None, '_is_chief': True, '_protocol': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_save_summary_steps': 100, '_keep_checkpoint_every_n_hours': 10000, '_save_checkpoints_secs': 600, '_tf_random_seed': None, '_master': '', '_log_step_count_steps': 100, '_save_checkpoints_steps': None, '_global_id_in_cluster': 0, '_model_dir': 'kmeans_estimator', '_keep_checkpoint_max': 5, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f5da64e4390>, '_task_id': 0, '_train_distribute': None, '_experimental_distribute': None}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into kmeans_estimator/model.ckpt.\n",
      "WARNING:tensorflow:Training with estimator made no steps. Perhaps input is empty or misspecified.\n",
      "INFO:tensorflow:Loss for final step: None.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-1\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 1135330.1, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 3 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 1135330.1.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-3\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 868553.25, step = 3\n",
      "INFO:tensorflow:Saving checkpoints for 5 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 868553.25.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-5\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 5 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 834369.6, step = 5\n",
      "INFO:tensorflow:Saving checkpoints for 7 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 834369.6.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-7\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 819098.9, step = 7\n",
      "INFO:tensorflow:Saving checkpoints for 9 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 819098.9.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-9\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 9 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 810747.9, step = 9\n",
      "INFO:tensorflow:Saving checkpoints for 11 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 810747.9.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-11\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 11 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 805639.7, step = 11\n",
      "INFO:tensorflow:Saving checkpoints for 13 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 805639.7.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-13\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 13 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 802136.0, step = 13\n",
      "INFO:tensorflow:Saving checkpoints for 15 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 802136.0.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-15\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 15 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 799587.8, step = 15\n",
      "INFO:tensorflow:Saving checkpoints for 17 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 799587.8.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-17\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 17 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 797632.3, step = 17\n",
      "INFO:tensorflow:Saving checkpoints for 19 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 797632.3.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-19\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 19 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 796076.44, step = 19\n",
      "INFO:tensorflow:Saving checkpoints for 21 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 796076.44.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-21\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 794811.6, step = 21\n",
      "INFO:tensorflow:Saving checkpoints for 23 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 794811.6.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-23\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 23 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 793762.75, step = 23\n",
      "INFO:tensorflow:Saving checkpoints for 25 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 793762.75.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-25\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 25 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 792873.1, step = 26\n",
      "INFO:tensorflow:Saving checkpoints for 27 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 792873.1.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-27\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 27 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 792104.25, step = 27\n",
      "INFO:tensorflow:Saving checkpoints for 29 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 792104.25.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-29\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 29 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 791429.1, step = 29\n",
      "INFO:tensorflow:Saving checkpoints for 31 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 791429.1.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-31\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 31 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 790833.25, step = 31\n",
      "INFO:tensorflow:Saving checkpoints for 33 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 790833.25.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-33\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 33 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 790303.75, step = 33\n",
      "INFO:tensorflow:Saving checkpoints for 35 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 790303.75.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-35\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 35 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 789831.94, step = 35\n",
      "INFO:tensorflow:Saving checkpoints for 37 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 789831.94.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-37\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 37 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 789408.94, step = 37\n",
      "INFO:tensorflow:Saving checkpoints for 39 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 789408.94.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-39\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 39 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 789027.75, step = 39\n",
      "INFO:tensorflow:Saving checkpoints for 41 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 789027.75.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-41\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 41 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 788682.0, step = 42\n",
      "INFO:tensorflow:Saving checkpoints for 43 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 788682.0.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-43\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 43 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 788366.4, step = 43\n",
      "INFO:tensorflow:Saving checkpoints for 45 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 788366.4.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-45\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 45 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 788074.25, step = 45\n",
      "INFO:tensorflow:Saving checkpoints for 47 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 788074.25.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-47\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 47 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 787803.0, step = 47\n",
      "INFO:tensorflow:Saving checkpoints for 49 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 787803.0.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-49\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 49 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 787552.06, step = 49\n",
      "INFO:tensorflow:Saving checkpoints for 51 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 787552.06.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-51\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 51 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 787320.6, step = 52\n",
      "INFO:tensorflow:Saving checkpoints for 53 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 787320.6.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-53\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 53 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 787105.4, step = 53\n",
      "INFO:tensorflow:Saving checkpoints for 55 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 787105.4.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-55\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 55 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 786905.0, step = 55\n",
      "INFO:tensorflow:Saving checkpoints for 57 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 786905.0.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-57\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 57 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:loss = 786716.5, step = 57\n",
      "INFO:tensorflow:Saving checkpoints for 59 into kmeans_estimator/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 786716.5.\n",
      "cluster centers = \n",
      "[[-6.395126   -0.6284118  -1.8389614   2.5162723  -0.08462896  0.2584047\n",
      "   0.3101617   0.1657284   0.38934284 -0.8458532 ]\n",
      " [-7.1962943   1.0342586  -0.6572838   0.02381766 -1.2975217  -0.21344638\n",
      "   1.4107537   0.14225866 -1.093288    0.618131  ]\n",
      " [-6.1652174   1.2391485   2.433958   -0.60400164 -0.41343993  0.14786275\n",
      "   0.65092695 -0.38034663 -0.6272439  -0.80905837]\n",
      " [-4.238364    2.345509   -1.5270541  -0.80450654  0.21260945 -1.0309664\n",
      "  -0.34272596 -0.9098434   0.26283526 -0.08501396]\n",
      " [-4.5334916   1.6444974  -0.5531736   0.07194325  0.19889057  1.4767112\n",
      "  -1.0294224   0.11387741 -0.33888492  0.5927328 ]\n",
      " [-7.051715   -4.6722074  -0.9717695  -0.15025206 -1.3253359  -1.2747681\n",
      "  -1.3595604  -0.6946098  -1.0158199   0.5483409 ]\n",
      " [-5.6364985   1.2949673   2.4596317  -0.15726888 -1.5233377  -1.3357512\n",
      "  -1.2730011   1.2839359   0.23872846 -0.83269256]\n",
      " [-7.381949   -1.496229   -0.83284456 -1.982439   -1.644541    1.0631865\n",
      "   0.36268845  0.07744081  1.2396921   0.10630859]\n",
      " [-5.366795   -0.32256594  2.408731    1.4642966   0.82164514 -0.09060739\n",
      "   0.24355662 -0.5768775   0.92056054  0.8244875 ]\n",
      " [-6.794806   -0.7407117  -0.11979345 -1.275778    2.5508614   0.16503784\n",
      "  -0.13511156  0.63348895 -0.25108165 -0.05336798]]\n",
      "WARNING:tensorflow:Input graph does not use tf.data.Dataset or contain a QueueRunner. That means predict yields forever. This is probably a mistake.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from kmeans_estimator/model.ckpt-59\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "shutil.rmtree(path = KMEANS_MODEL_DIR, ignore_errors = True) # start fresh each time\n",
    "\n",
    "def input_fn():\n",
    "  return tf.train.limit_epochs(\n",
    "    tensor = tf.convert_to_tensor(\n",
    "      value = x_train_pca_arr, \n",
    "      dtype = tf.float32), \n",
    "    num_epochs = 1)\n",
    "\n",
    "num_clusters = 10\n",
    "kmeans = tf.contrib.factorization.KMeansClustering(\n",
    "  num_clusters = num_clusters, \n",
    "  model_dir = KMEANS_MODEL_DIR,\n",
    "  initial_clusters = tf.contrib.factorization.KMeansClustering.KMEANS_PLUS_PLUS_INIT,\n",
    "  use_mini_batch = True)\n",
    "\n",
    "# Train\n",
    "num_iterations = 30\n",
    "previous_centers = None\n",
    "for _ in range(num_iterations):\n",
    "  kmeans.train(input_fn = input_fn)\n",
    "  cluster_centers = kmeans.cluster_centers()\n",
    "  previous_centers = cluster_centers\n",
    "print(\"cluster centers = \\n{}\".format(cluster_centers))\n",
    "\n",
    "# Map the input points to their clusters\n",
    "cluster_indices = list(kmeans.predict_cluster_index(input_fn = input_fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster_indices_arr = np.array(object = cluster_indices)\n",
    "cluster_indices_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 10)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "point_clusters_arr = cluster_centers[cluster_indices_arr, :]\n",
    "point_clusters_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 10)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "squared_error_arr = (x_train_pca_arr - point_clusters_arr)**2\n",
    "squared_error_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "squared_euclidean_distance = np.sum(a = squared_error_arr, axis = 1)\n",
    "squared_euclidean_distance.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster_id</th>\n",
       "      <th>squared_euclidean_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>9.698139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>12.847395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>13.683814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7.027799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>4.790666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cluster_id  squared_euclidean_distance\n",
       "0           1                    9.698139\n",
       "1           7                   12.847395\n",
       "2           8                   13.683814\n",
       "3           3                    7.027799\n",
       "4           2                    4.790666"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans_df = pd.DataFrame({\"cluster_id\": cluster_indices_arr, \n",
    "                          \"squared_euclidean_distance\": squared_euclidean_distance})\n",
    "kmeans_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>squared_euclidean_distance</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>7587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7786</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            squared_euclidean_distance\n",
       "cluster_id                            \n",
       "0                                 7443\n",
       "1                                 5461\n",
       "2                                 5454\n",
       "3                                 6929\n",
       "4                                 7485\n",
       "5                                 3578\n",
       "6                                 4271\n",
       "7                                 4006\n",
       "8                                 7587\n",
       "9                                 7786"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans_df.groupby(\"cluster_id\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>squared_euclidean_distance</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.651371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13.524157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.011460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.157743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.979157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>15.743523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10.246287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>16.385509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12.881548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18.233294</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            squared_euclidean_distance\n",
       "cluster_id                            \n",
       "0                            14.651371\n",
       "1                            13.524157\n",
       "2                            11.011460\n",
       "3                             9.157743\n",
       "4                             9.979157\n",
       "5                            15.743523\n",
       "6                            10.246287\n",
       "7                            16.385509\n",
       "8                            12.881548\n",
       "9                            18.233294"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans_df.groupby(\"cluster_id\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cluster_id       \n",
       "0           16825    1.989303\n",
       "            7508     2.487936\n",
       "            16783    2.785687\n",
       "            12874    3.033547\n",
       "            7396     3.227566\n",
       "1           23143    1.664769\n",
       "            36927    1.868385\n",
       "            41235    2.096154\n",
       "            50925    2.106042\n",
       "            56141    2.164360\n",
       "2           53655    1.745137\n",
       "            27041    1.840623\n",
       "            51757    1.871719\n",
       "            36867    1.930790\n",
       "            2217     1.958854\n",
       "3           43121    1.112363\n",
       "            22283    1.657932\n",
       "            43109    1.755380\n",
       "            28361    1.857489\n",
       "            7329     1.910113\n",
       "4           42104    1.589759\n",
       "            21868    1.593555\n",
       "            35828    1.790105\n",
       "            14162    1.906652\n",
       "            2954     2.025714\n",
       "5           31095    1.750447\n",
       "            22706    2.479543\n",
       "            50958    2.554381\n",
       "            53881    2.598932\n",
       "            9197     2.809373\n",
       "6           40211    1.797966\n",
       "            16369    1.983479\n",
       "            43019    2.063008\n",
       "            9676     2.076820\n",
       "            33988    2.158072\n",
       "7           38380    2.294816\n",
       "            42863    2.836855\n",
       "            59269    3.161669\n",
       "            51758    3.405574\n",
       "            53522    3.654172\n",
       "8           11628    1.132438\n",
       "            18786    1.730492\n",
       "            3260     1.863786\n",
       "            16090    1.921478\n",
       "            52572    1.988268\n",
       "9           41207    3.295390\n",
       "            25770    3.299531\n",
       "            17626    3.339903\n",
       "            36184    3.353349\n",
       "            35825    3.692197\n",
       "Name: squared_euclidean_distance, dtype: float32"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans_df.groupby(\"cluster_id\")[\"squared_euclidean_distance\"].nsmallest(n = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000,)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "closest_indices = np.array(\n",
    "  object = kmeans_df.groupby(\"cluster_id\")[\"squared_euclidean_distance\"].nsmallest(n = 100).index.get_level_values(1))\n",
    "closest_indices.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try semi-supervised again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "semi_supervised_labeled_x_train_original_arr = x_train[closest_indices]\n",
    "semi_supervised_labeled_y_train_original_arr = y_train[closest_indices]\n",
    "semi_supervised_unlabeled_x_train_original_arr = x_train[np.isin(\n",
    "  element = np.arange(number_of_train_examples), \n",
    "  test_elements = closest_indices, \n",
    "  assume_unique = True, \n",
    "  invert = True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "semi_supervised_labeled_x_train_original_arr.shape = (1000, 28, 28)\n",
      "semi_supervised_labeled_y_train_original_arr.shape = (1000, 10)\n",
      "semi_supervised_unlabeled_x_train_original_arr.shape = (59000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "print(\"semi_supervised_labeled_x_train_original_arr.shape = {}\".format(semi_supervised_labeled_x_train_original_arr.shape))\n",
    "print(\"semi_supervised_labeled_y_train_original_arr.shape = {}\".format(semi_supervised_labeled_y_train_original_arr.shape))\n",
    "print(\"semi_supervised_unlabeled_x_train_original_arr.shape = {}\".format(semi_supervised_unlabeled_x_train_original_arr.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "loop_counter = 0, number_of_labeled_examples = 1000, number_of_unlabeled_examples = 59000\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'random_shuffle_queue_DequeueMany:1' shape=(32, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"random_shuffle_queue_DequeueMany:2\", shape=(32, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "train\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(32, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(32, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(32,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.3920372017156772, step = 0\n",
      "INFO:tensorflow:global_step/sec: 400.333\n",
      "INFO:tensorflow:loss = 0.0738389307699354, step = 100 (0.252 sec)\n",
      "INFO:tensorflow:global_step/sec: 455.854\n",
      "INFO:tensorflow:loss = 0.04258015815357359, step = 200 (0.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 446.27\n",
      "INFO:tensorflow:loss = 0.02258214835423311, step = 300 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 425.347\n",
      "INFO:tensorflow:loss = 0.009491640485282206, step = 400 (0.239 sec)\n",
      "INFO:tensorflow:global_step/sec: 440.492\n",
      "INFO:tensorflow:loss = 0.015361745249750737, step = 500 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 449.829\n",
      "INFO:tensorflow:loss = 0.010533202415866704, step = 600 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 440.982\n",
      "INFO:tensorflow:loss = 0.008551840108789937, step = 700 (0.228 sec)\n",
      "INFO:tensorflow:global_step/sec: 433.646\n",
      "INFO:tensorflow:loss = 0.00651606237399625, step = 800 (0.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 444.681\n",
      "INFO:tensorflow:loss = 0.004436112992659646, step = 900 (0.222 sec)\n",
      "INFO:tensorflow:global_step/sec: 450.126\n",
      "INFO:tensorflow:loss = 0.002297883637706603, step = 1000 (0.222 sec)\n",
      "INFO:tensorflow:global_step/sec: 454.004\n",
      "INFO:tensorflow:loss = 0.003859210935698811, step = 1100 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 435.908\n",
      "INFO:tensorflow:loss = 0.003363932910097761, step = 1200 (0.226 sec)\n",
      "INFO:tensorflow:global_step/sec: 456.016\n",
      "INFO:tensorflow:loss = 0.003174765574021, step = 1300 (0.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 452.861\n",
      "INFO:tensorflow:loss = 0.0028088958466904934, step = 1400 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 449.741\n",
      "INFO:tensorflow:loss = 0.002802493667370615, step = 1500 (0.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 453.991\n",
      "INFO:tensorflow:loss = 0.002231987488621738, step = 1600 (0.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 445.735\n",
      "INFO:tensorflow:loss = 0.001047506867606553, step = 1700 (0.224 sec)\n",
      "INFO:tensorflow:global_step/sec: 457.902\n",
      "INFO:tensorflow:loss = 0.0014828329034901676, step = 1800 (0.216 sec)\n",
      "INFO:tensorflow:global_step/sec: 453.007\n",
      "INFO:tensorflow:loss = 0.00011045118590306294, step = 1900 (0.221 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.001304640192603684.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"fifo_queue_DequeueUpTo:2\", shape=(?, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "eval\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-15T23:59:52Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-15-23:59:53\n",
      "INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.7597, global_step = 2000, loss = 1.2585862\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "None\n",
      "mode = \n",
      "infer\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "loop_counter = 1, number_of_labeled_examples = 30704, number_of_unlabeled_examples = 29296\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'random_shuffle_queue_DequeueMany:1' shape=(32, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"random_shuffle_queue_DequeueMany:2\", shape=(32, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "train\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(32, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(32, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(32,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.014489982386799865, step = 2000\n",
      "INFO:tensorflow:global_step/sec: 364.615\n",
      "INFO:tensorflow:loss = 0.007301093863149857, step = 2100 (0.276 sec)\n",
      "INFO:tensorflow:global_step/sec: 417.327\n",
      "INFO:tensorflow:loss = 0.009420173575188845, step = 2200 (0.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 419.05\n",
      "INFO:tensorflow:loss = 0.007188501770620521, step = 2300 (0.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 415.403\n",
      "INFO:tensorflow:loss = 0.010242672632107427, step = 2400 (0.239 sec)\n",
      "INFO:tensorflow:global_step/sec: 415.22\n",
      "INFO:tensorflow:loss = 0.004882665689894688, step = 2500 (0.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 410.786\n",
      "INFO:tensorflow:loss = 0.012743466159321117, step = 2600 (0.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 410.827\n",
      "INFO:tensorflow:loss = 0.010767896674761199, step = 2700 (0.246 sec)\n",
      "INFO:tensorflow:global_step/sec: 419.712\n",
      "INFO:tensorflow:loss = 0.012821899567417038, step = 2800 (0.238 sec)\n",
      "INFO:tensorflow:global_step/sec: 410.055\n",
      "INFO:tensorflow:loss = 0.012072403138855398, step = 2900 (0.241 sec)\n",
      "INFO:tensorflow:global_step/sec: 414.355\n",
      "INFO:tensorflow:loss = 0.007228608622058456, step = 3000 (0.241 sec)\n",
      "INFO:tensorflow:global_step/sec: 415.249\n",
      "INFO:tensorflow:loss = 0.009648972516631165, step = 3100 (0.241 sec)\n",
      "INFO:tensorflow:global_step/sec: 412.85\n",
      "INFO:tensorflow:loss = 0.010943552106717352, step = 3200 (0.242 sec)\n",
      "INFO:tensorflow:global_step/sec: 412.755\n",
      "INFO:tensorflow:loss = 0.012223714447477226, step = 3300 (0.243 sec)\n",
      "INFO:tensorflow:global_step/sec: 401.107\n",
      "INFO:tensorflow:loss = 0.0016599531763643867, step = 3400 (0.252 sec)\n",
      "INFO:tensorflow:global_step/sec: 403.388\n",
      "INFO:tensorflow:loss = 0.006771439153691364, step = 3500 (0.246 sec)\n",
      "INFO:tensorflow:global_step/sec: 410.968\n",
      "INFO:tensorflow:loss = 0.008963326755513881, step = 3600 (0.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 408.838\n",
      "INFO:tensorflow:loss = 0.007872207025242384, step = 3700 (0.248 sec)\n",
      "INFO:tensorflow:global_step/sec: 436.097\n",
      "INFO:tensorflow:loss = 0.004315283459277182, step = 3800 (0.226 sec)\n",
      "INFO:tensorflow:global_step/sec: 453.726\n",
      "INFO:tensorflow:loss = 0.014596631408547028, step = 3900 (0.220 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4000 into semi_supervised_trained/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.006888440293884521.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "Tensor(\"fifo_queue_DequeueUpTo:2\", shape=(?, 10), dtype=float64, device=/device:CPU:0)\n",
      "mode = \n",
      "eval\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-15T23:59:59Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-4000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-16-00:00:00\n",
      "INFO:tensorflow:Saving dict for global step 4000: accuracy = 0.7592, global_step = 4000, loss = 1.2668405\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 4000: semi_supervised_trained/model.ckpt-4000\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "features = \n",
      "{'image': <tf.Tensor 'fifo_queue_DequeueUpTo:1' shape=(?, 28, 28) dtype=float64>}\n",
      "labels = \n",
      "None\n",
      "mode = \n",
      "infer\n",
      "params = \n",
      "{'ksize1': 5, 'batch_norm': False, 'ksize2': 5, 'learning_rate': 0.01, 'train_batch_size': 100, 'train_steps': 1000, 'nfil2': 20, 'nfil1': 10, 'dprob': 0.1, 'model': 'linear'}\n",
      "ylogits = \n",
      "Tensor(\"dense/BiasAdd:0\", shape=(?, 10), dtype=float64)\n",
      "probabilities = \n",
      "Tensor(\"Softmax:0\", shape=(?, 10), dtype=float64)\n",
      "class_ids = \n",
      "Tensor(\"Cast:0\", shape=(?,), dtype=uint8)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from semi_supervised_trained/model.ckpt-4000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "shutil.rmtree(path = SEMI_SUPERVISED_MODEL_DIR, ignore_errors = True) # start fresh each time\n",
    "\n",
    "semi_supervised_labeled_x_train_arr = semi_supervised_labeled_x_train_original_arr\n",
    "semi_supervised_labeled_y_train_arr = semi_supervised_labeled_y_train_original_arr\n",
    "semi_supervised_unlabeled_x_train_arr = semi_supervised_unlabeled_x_train_original_arr\n",
    "\n",
    "new_labeled_x_train_arr = np.zeros([1])\n",
    "\n",
    "accuracy = 0.000001\n",
    "old_accuracy = 0.0\n",
    "\n",
    "loop_counter = 0\n",
    "while semi_supervised_unlabeled_x_train_arr.shape[0] > 0 and new_labeled_x_train_arr.shape[0] > 0 and accuracy > old_accuracy:\n",
    "  print(\"\\nloop_counter = {}, number_of_labeled_examples = {}, number_of_unlabeled_examples = {}\\n\".format(loop_counter, semi_supervised_labeled_x_train_arr.shape[0], semi_supervised_unlabeled_x_train_arr.shape[0]))\n",
    "  # Train on currently labeled data\n",
    "  train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x = {\"image\": semi_supervised_labeled_x_train_arr}, \n",
    "    y = semi_supervised_labeled_y_train_arr, \n",
    "    batch_size = 32, \n",
    "    num_epochs = None, \n",
    "    shuffle = True)\n",
    "\n",
    "  semi_supervised_estimator.train(\n",
    "    input_fn = train_input_fn, \n",
    "    steps = 2000)\n",
    "\n",
    "\n",
    "  # Check evaluation metrics on held out evaluation set now that training is over\n",
    "  eval_metrics = semi_supervised_estimator.evaluate(\n",
    "    input_fn = eval_input_fn, \n",
    "    steps = None)\n",
    "  \n",
    "  old_accuracy = accuracy\n",
    "  accuracy = eval_metrics[\"accuracy\"]\n",
    "\n",
    "  # Now predict from the unlabeled set\n",
    "  predict_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x = {\"image\": semi_supervised_unlabeled_x_train_arr}, \n",
    "    y = None, \n",
    "    batch_size = 512, \n",
    "    num_epochs = 1, \n",
    "    shuffle = False)\n",
    "\n",
    "  predictions = [prediction \n",
    "                 for prediction in semi_supervised_estimator.predict(\n",
    "                   input_fn = predict_input_fn)]\n",
    "\n",
    "  # Get the probabilities from the prediction list generated from the estimator\n",
    "  probabilities = np.array(object = [prediction[\"probabilities\"] \n",
    "                                     for prediction in predictions])\n",
    "\n",
    "  # Check if our predictions are above the confidence threshold\n",
    "  confidence_condition = np.amax(a = probabilities, axis = 1) > confidence_threshold\n",
    "\n",
    "  # Create array of the confidently prediction examples combining their features with the predicted probabilities\n",
    "  new_labeled_x_train_arr = semi_supervised_unlabeled_x_train_arr[confidence_condition]\n",
    "  new_labeled_y_train_arr = probabilities[confidence_condition]\n",
    "\n",
    "  semi_supervised_labeled_x_train_arr = np.concatenate(\n",
    "    seq = [semi_supervised_labeled_x_train_arr, new_labeled_x_train_arr], axis = 0)\n",
    "  semi_supervised_labeled_y_train_arr = np.concatenate(\n",
    "    seq = [semi_supervised_labeled_y_train_arr, new_labeled_y_train_arr], axis = 0)\n",
    "\n",
    "  # Remove the confident predictions leaving only the unconfident predictions to go another round through the loop\n",
    "  semi_supervised_unlabeled_x_train_arr = semi_supervised_unlabeled_x_train_arr[~confidence_condition]\n",
    "  \n",
    "  loop_counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
